const puppeteer = require('puppeteer');
const MongoClient = require('mongodb').MongoClient;
const ObjectId = require('mongodb').ObjectId;

const mongo_url = "mongodb://egg_cnode:bojun1234@124.223.34.189:27127/egg_cnode?authSource=egg_cnode";


const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

async function main() {
  browser = await puppeteer.launch({
    headless: false,
    ignoreHTTPSErrors: true,
    args: ['--no-sandbox', `--proxy-server=http://u5329.b5.t.16yun.cn:6460`]
  });
  const page = (await browser.pages())[0];
  await page.authenticate({'username': '16GADNHJ', 'password': '936441'});
  await page.setViewport({width: 1500, height: 0});

  const client = await MongoClient.connect(mongo_url, {useNewUrlParser: true, useUnifiedTopology: true});
  const db = client.db('egg_cnode');

  let tab = 'knowledge';
  let sub_tab = '果壳';
  const fileName = `${tab}_${sub_tab}.json`;
  const source = '果壳' + (new Date(+new Date() + 1000 * 3600 * 8)).toISOString().split('T')[0];
  const real_author_name = '科学人';


  page.on('pageerror', msg => {
    console.log(msg);
  });

  for (let i = 1; i < 10000; i++) {
    let totalTopics = [];
    // let url = `https://www.guokr.com/apis/minisite/article.json?retrieve_type=by_wx&channel_key=pac&offset=${i * 20}&limit=20`;
    let url = `https://www.guokr.com/apis/minisite/article.json?retrieve_type=by_wx&channel_key=predator&offset=${i * 20}&limit=20`;
    await page.goto(url, {waitUntil: 'domcontentloaded'});
    let content = await page.content();

    content = content.replace('<html><head></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">', '')
      .replace('</pre></body></html>', '').trim();
    let objs = JSON.parse(content);
    objs = objs.result;
    for (let obj of objs) {
      let topic = {
        title: obj.title,
        content: `https://www.guokr.com/article/${obj.id}`,
        intro: obj.summary,
        pic: obj.image_portrait,
        original_url: `https://www.guokr.com/article/${obj.id}`,
        tab: 'knowledge',
        sub_tab: '果壳',
        source,
        real_author_name: obj.author.nickname,
        author_id: '61dda2a097ceba9ea3ea8ca2',
        author_name: '果壳',
        author_avatar: '/public/images/guokr.png',
        is_public: true,
        deleted: false,
        create_at: new Date(),
        update_at: new Date(),
        last_reply_at: new Date(),
        top: false,
      }
      if (!topic.pic || !topic.pic.startsWith('http')) {
        topic.pic = obj.small_image;
      }
      if (topic.pic == 'https://3-im.guokr.com/Fop276nTC9MXMtM8i1c6XjP0Cb60xlOMo937M1lL0wG0AAAAjAAAAFBO.png') {
        if (obj.thumb_image) {
          let existTopic = await db.collection('topics').findOne({title: topic.title, author_name: '果壳'});
          let result = await db.collection('topics').updateOne({_id: existTopic._id}, {
            $set: {
              pic: obj.thumb_image
            }
          });
          console.log(result.result, i);
        } else {
          console.log(obj);
        }
      }
    }
  }
};


main();


